package com.xiaotu.spider.project.pageprocessor.list;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang.StringUtils;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import com.google.gson.Gson;
import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.FileUtil;
import com.xiaotu.common.util.HttpUtils;
import com.xiaotu.common.util.KeywordEncode;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.SepratorUtil;
import com.xiaotu.common.util.StringUtil;
import com.xiaotu.spider.AUrlPageProcessor;

import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;

/**
 * @类名 YouKuCommentListPageProcessor
 * @日期 2017年8月29日
 * @作者 王艳龙
 * @功能  优酷爬取类--TV 短评
 */
@Component("YouKuCommentListPageProcessor")
@Scope("prototype")
public class YouKuCommentListPageProcessor extends AUrlPageProcessor
{
	
    private String fileName; //读取tvName文件路径
    private String YouKuComment; //优酷短评
    
    @Override
    protected String[] getStartRequestUrls()
    {
        List<JSONObject> targetList = readFileColumn(fileName); //读取excel电影名
        List<String> linkList = new ArrayList<String>();
        for (JSONObject target : targetList)
        {
            linkList.add("http://www.soku.com/search_video/q_"
				+ KeywordEncode.encode(target.getString("tvName")));
        }
        String[] urlArr = new String[linkList.size()];
        linkList.toArray(urlArr);
        return urlArr;

    }

    @Override
    public void initRunParams(Map<String, Object> map)
    {
        if (map.get("fileName") == null)
            throw new SpiderException(
                    SpiderExceptionCode.PageProcessorParameterErrorException);
        fileName = map.get("fileName") + StringUtils.EMPTY;

        if (map.get("YouKuComment") == null)
            throw new SpiderException(
                    SpiderExceptionCode.PageProcessorParameterErrorException);
        YouKuComment = map.get("YouKuComment") + StringUtils.EMPTY;
    }

    @Override
    public String getPageDataType(Page page)
    {
    	String url = page.getUrl().toString();
		if (url.contains("http://www.soku.com/search_video/q_"))
			return Constants.PageDataType.SEARCH_LIST;
		else if (url.contains("http://cps.youku.com/redirect.html"))
			return Constants.PageDataType.TV_RECORD;
		else if (url.contains("/comment"))
			return Constants.PageDataType.TV_COMMENT;
		else
			return Constants.PageDataType.SEARCH_LIST;
    }

    @Override
    public void process(Page page)
    {
    	Map<String, Object> tvMap = page.getRequest().getExtras();
		switch (this.getPageDataType(page))
		{
			case Constants.PageDataType.SEARCH_LIST:
				this.searchListProcess(page, tvMap);
				break;
			case Constants.PageDataType.TV_RECORD:
				this.tvRecordProcess(page, tvMap);
				break;
			case Constants.PageDataType.TV_COMMENT:
				this.commentProcess(page, tvMap);
				break;
			default:
				break;
		}
    }

    private void searchListProcess(Page page, final Map<String, Object> tvMap)
	{
		Html html = page.getHtml();
		//网大名称
		String movieName =html.xpath("//div[@class='s_act site14']/a/@_log_title").toString();
		if(StringUtils.isNotEmpty(movieName)){
			tvMap.put("movieName", movieName);
		}
		// 链接标签
		String linkUrl = html.xpath("//div[@class='s_act site14']/a/@href").toString();
		if(StringUtils.isNotEmpty(linkUrl)){
			PageUtils.addTargetRequest(page,linkUrl,tvMap);
		}
	}
    
    private void tvRecordProcess(Page page, final Map<String, Object> tvMap)
   	{
   		Html html = page.getHtml();
   		String objectId = getSrings(html.toString(), Pattern.quote("videoId:\"") +
                "(.*?)" + Pattern.quote("\", videoId2"));
   		if(StringUtils.isNotEmpty(objectId)){
   			//获取总页数
   			String jsonStr = HttpUtils.httpGetJSONP("http://p.comments.youku.com/ycp/comment/pc/commentList?jsoncallback=n_commentList&app=100-DDwODVkv"
						+ "&objectId="+objectId+""
						+ "&objectType=1&listType=0&currentPage=1&pageSize=30"
						+ "&sign=f3d2b8c10aae626ea025914972bc5f65&time=1503989886");
   			jsonStr = jsonStr.substring(jsonStr.indexOf("{"));
   			jsonStr = jsonStr.substring(0, jsonStr.lastIndexOf("}") + 1);
   			JSONObject C_json = JSONObject.fromObject(jsonStr);
   			String totalPage = C_json.getJSONObject("data").getString("totalPage");
   			int pageSize =Integer.valueOf(totalPage);
   			System.out.println(pageSize);
   			
   			for (int i = 1; i <= pageSize; i++)
   			{
   				// 添加短评链接
   				PageUtils.addTargetRequest(page,
   						"http://p.comments.youku.com/ycp/comment/pc/commentList?jsoncallback=n_commentList&app=100-DDwODVkv"
   								+ "&objectId="+objectId+""
   								+ "&objectType=1&listType=0&currentPage="+i+"&pageSize=30"
   								+ "&sign=f3d2b8c10aae626ea025914972bc5f65&time=1503989886",
   								tvMap);
   			}
   		}
   	}
	
	private void commentProcess(Page page, Map<String, Object> tvMap)
	{
		System.out.println(page.getHtml().toString());
		String jsonStr = page.getRawText();
		// format json
		jsonStr = jsonStr.substring(jsonStr.indexOf("{"));
		
		jsonStr = jsonStr.substring(0, jsonStr.lastIndexOf("}") + 1);
		jsonStr =jsonStr.substring(jsonStr.indexOf("data\":")+6, jsonStr.length());
		
		JSONObject C_json = JSONObject.fromObject(jsonStr);
		
		if (!C_json.containsKey("comment"))
			throw new SpiderException(
					SpiderExceptionCode.DataInvalideException);
		JSONArray comments = C_json.getJSONArray("comment");
		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm");
		
		for (int i = 0; i < comments.size(); i++)
		{
			JSONObject comment = comments.getJSONObject(i);
			
			if (!comment.containsKey("content"))
				continue;
			
			if (!comment.containsKey("createTime")
					|| StringUtils.isEmpty(comment.getString("createTime")))
				continue;
			
			if (!comment.containsKey("user"))
				continue;
			Date data = new Date(Long.valueOf(comment.getString("createTime")));
			String pubtime =sdf.format(data);
			String line = StringUtils.join(new String[]{StringUtil.nullToString(page.getRequest().getExtra("type")),
					KeywordEncode.decode((String)tvMap.get("movieName")), 
					comment.getJSONObject("user").getString("userName").replace("'", "''"), 
					pubtime, 
					comment.getString("content").replace("'", "''")}, SepratorUtil
	                .SEP_PIPE);
	        System.out.println(line);
	        FileUtil.writeFile(YouKuComment, line + SepratorUtil.LINE_SEPRATOR);
		}
	}
    /**
     * 读取文件指定列
     *
     * @param fileName 文件名称
     * @return 行列表
     */
    public static List<JSONObject> readFileColumn(String fileName)
    {
        Map<String, Object> map = new HashMap<String, Object>();
        Workbook wb = null;
        String json;
        List<JSONObject> list = new ArrayList<JSONObject>();
        try
        {
            InputStream inputStream = new FileInputStream(fileName);
            if (fileName.endsWith("xls"))
            {
                wb = new HSSFWorkbook(inputStream); // 解析xls格式
            } else if (fileName.endsWith("xlsx"))
            {
                wb = new XSSFWorkbook(inputStream); // 解析xlsx格式
            }
            Sheet sheet = wb.getSheetAt(0); // 第一个工作表

            int firstRowNum = 1;
            int lastRowNum = sheet.getLastRowNum();

            Row row = null;
            Cell cell_a1 = null;
            for (int i = firstRowNum; i <= lastRowNum; i++)
            {
                row = sheet.getRow(i);          //取得第i行
                cell_a1 = row.getCell(0);      //取得i行的第一列
                if(row.getCell(0)!=null){
                	cell_a1.setCellType(Cell.CELL_TYPE_STRING);
                    String cellValue1 = cell_a1.getStringCellValue();
                    map.put("tvName", cellValue1);
               }
                json = new Gson().toJson(map);
                JSONObject jsStr = JSONObject.fromObject(json);
                list.add(jsStr);
            }

        }
        catch (FileNotFoundException e)
        {
            e.printStackTrace();
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }
        return list;
    }
    
    /**
     * 获取多个目标字符串
     */
    public static String getSrings(String input, String regex)
    {
    	String result=null;
        Pattern pat = Pattern.compile(regex, Pattern.DOTALL);
        Matcher mat = pat.matcher(input);
        while (mat.find())
        {
            result = mat.group(1);
        }
        return result;
    }
    
    @Override
    protected String getSpiderType()
    {
        return "YouKuCommentList";
    }
}
